import scrapy
from ..items import CbcItem
from selenium import webdriver
from selenium.webdriver import FirefoxOptions
from threading import Thread, Lock
class ChinaSpider(scrapy.Spider):
    name = 'toronto'
    allowed_domains = ['https://www.thestar.com/']
    start_urls = ['https://www.thestar.com/search.html?&q=China&sortby=publishdate&sortdirection=desc']
    count = 316
    def __init__(self):
        super(ChinaSpider, self).__init__(name='toronto')
        option = FirefoxOptions()
        option.headless = True
        self.driver = webdriver.Firefox(options=option)
#337
    def parse(self, response):

        tr_list = response.xpath("//div[@class='c-search-results-item']")
        for i in range(1):
            tr= tr_list[i]
            item = CbcItem()
            href = tr.xpath("./a/@href").extract_first()
            item['herf'] = 'https://www.thestar.com' + href
            item['publish_date'] = tr.xpath("./div/text()").extract_first()
            item["title"] = tr.xpath("./a/div[@class='c-mediacard__content']/h3/span/text()").extract_first()
            item["brief"] = ""
            yield scrapy.Request(
                item['herf'],
                callback=self.parse_detail,
                meta={"item": item},
                dont_filter=True
            )
        self.count = self.count + 1
        if (self.count <= 337):
            print(self.count)
            yield scrapy.Request(
                "https://www.thestar.com/search.html?q=China&sortby=publishdate&sortdirection=desc&pagenum=" + str(self.count),
                callback=self.parse,
                dont_filter=True
            )
    def parse_detail(self, response):
        item = response.meta['item']
        item["content"] = response.xpath("//*[@id='body-content']/div/div/div[1]/div[2]/div/p/text()").extract()
        item["content"] = "".join(item["content"])
        yield(item)